# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 
# Message:

from untils.MysqlTools import MysqlTools
from untils.requestsTools import requestsTools
from setting import *
import redis
import traceback
import random, time, os
from lxml import etree

class PriceSpider:

    def __init__(self):
        self.sqlDb = MysqlTools(db_config=sqlDb, type='dict', auto_commit=1000)
        htmlPath = os.getcwd() + os.sep + '..' + os.sep + 'bmwpriceHtml'
        if not os.path.exists(htmlPath):
            os.makedirs(htmlPath)
        self.requestsTools = requestsTools(htmlPath)
        self.requestsTools.headers = HEADERS
        self.requestsTools.cookies = COOKIES
        self.r = redis.Redis(**redisDb).pipeline()
        self.sleep_num = 0

    def get_allPid(self):
        sql = '''select distinct `pid`, `brand_code` from `{}`'''.format(selectTb)
        try:
            data = self.sqlDb.select_all(sql)
        except:
            print(traceback.format_exc())
            data = []
        return data

    def get_priceInfo(self, search_pid):
        price_url = "https://www.dwp.bmw-brilliance.cn/sap(bD16aCZjPTEwMA==)/bc/bsp/bmw/gis_tp_par_dev/spp05_material_details.htm?if_material={}"
        price_url = price_url.format(search_pid)
        price_responseInfo = self.requestsTools.parseUrl(url=price_url, fileName=search_pid, toFile=toFile)
        responseContent = price_responseInfo.get("content")
        if responseContent:
            priceHtml = etree.HTML(responseContent)
            price = priceHtml.xpath(u'//td[contains(text(),"零售价")]/following-sibling::td[1]/text()')
            stock_price = priceHtml.xpath(u'//td[contains(text(),"净值 (Stock)")]/following-sibling::td[1]/text()')
            rush_price = priceHtml.xpath(u'//td[contains(text(),"净值 (Rush)")]/following-sibling::td[1]/text()')

            price = price[0] if len(price) > 0 else ""
            stock_price = stock_price[0] if len(stock_price) > 0 else ""
            rush_price = rush_price[0] if len(rush_price) > 0 else ""

            # 数据清洗
            price = float(price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))
            stock_price = float(
                stock_price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))
            rush_price = float(
                rush_price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))
            if price == 0.0:
                return {}
            print(search_pid, price, stock_price, rush_price)
            last_data = dict(
                price=str(price),
                stock_price=str(stock_price),
                rush_price=str(rush_price)
            )
            return last_data
        else:
            print("{} 没有获取到价格信息".format(search_pid))
            exit(0)

    def clean2db(self, priceInfo):
        pid = priceInfo.get('pid')
        brand = priceInfo.get("brand")
        price = priceInfo.get('price').replace('.0', '')
        stock_price = priceInfo.get('stock_price').replace('.0', '')
        rush_price = priceInfo.get('rush_price').replace('.0', '')
        if rush_price == '0':
            return
        data_dict = dict(
            source_db='price_detail',
            brand_code=brand,
            firm_code='bmw',
            pid=pid,
            price=int(price),
            purchase_price=0,
            purchase_price_eot=int(stock_price),
            remark='',
            available=1,
            extra='',
            has_article=0,
            create_time='now()'
        )
        sql_tail = '''on duplicate key update `purchase_price_eot`=values(`purchase_price_eot`), `price`=values(`price`)'''
        table = insertTb
        sql = self.sqlDb.get_insert_sql(table, data_dict) + sql_tail
        try:
            self.sqlDb.execute_sql(sql)
        except Exception as e:
            print(sql)
            raise e


    def getData(self, pid, brand):
        # 构建最终pid
        try:
            search_pid = str("%018d" % int(pid))
        except:
            search_pid = str(pid)

        priceInfo = self.get_priceInfo(search_pid)
        if not priceInfo:
            return
        priceInfo["pid"] = pid
        priceInfo["brand"] = brand

        # 最终数据清洗入库
        self.clean2db(priceInfo)

    def clean_redis(self):
        self.r.delete('bmw_price_set')
        self.r.execute()

    def run(self):
        allPid = self.get_allPid()
        for pidInfo in allPid:
            pid = pidInfo.get("pid")
            brand = pidInfo.get("brand_code")

            # 断点代码块
            self.r.sismember('bmw_price_set', pid)
            if self.r.execute()[0]:
                continue
            self.r.sadd('bmw_price_set', pid)
            # 间歇代码块
            if self.sleep_num >= 10:
                sleep_time = random.randint(3, 5)
                # print "连续抓取10次, 间歇%d秒" % sleep_time
                time.sleep(sleep_time)
                self.sleep_num = 0  # 间歇完毕,重置为0
            self.sleep_num += 1
            self.getData(pid, brand)
            self.r.execute()
        self.sqlDb.commit_sql()

def main():
    tools = PriceSpider()
    try:
        tools.clean_redis()
        tools.run()
    except:
        print(traceback.format_exc())

if __name__ == '__main__':
    main()